import urllib2
import datetime
import lxml.html
import cookielib
import httplib
import re
from time import sleep

SOPCAST_URLS_FILE_PATH = u'sopcast-urls-{0:s}.m3u'.format(
    datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d_%H%M%S'))

httplib.HTTPConnection.debuglevel = 1

cookiejar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1018.0 Safari/535.19'),
                    ('Referer', 'http://translate.google.com/')
]
urllib2.install_opener(opener)

html_data = opener.open(START_PAGE_URL).read()

doc = lxml.html.fromstring(html_data)

links = doc.xpath('//a[@onclick]')

popup_links = []
sopcast_links = []


for link in links:
    attr = link.get('onclick')
    match = re.search("^.+window.open\((\'|\")(?P<URL>[^\'\"]+)", attr)

    if match:
        sopcast_links_file = open(SOPCAST_URLS_FILE_PATH, 'a')
        url = match.group('URL')

        title = link.xpath('text()')[0].strip()
        print "%s [%s]" % (title,  url)

        if url and url.startswith('http'):
            sopcast_links_file.write('# TITLE %s\n' % title)
            sopcast_links_file.write('# page URL %s\n' % url)

            popup_links.append(url)
            sleep(5)

            print 'opening url %s...' % url
            popup_html = opener.open(url).read()
            popup_doc = lxml.html.fromstring(popup_html)
            sopParam = popup_doc.xpath("//param[starts-with(@value, 'sop://')]")

            if len(sopParam):
                sopUrl = sopParam[0].get('value')
                print sopUrl
                sopcast_links_file.write(sopUrl + '\n')
                # //a[starts-with(@href, "buy.php/")]
        else:
            print '\tdiscarding URL', url
        sopcast_links_file.close()
